{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1</th>\n",
       "      <th>feat_2</th>\n",
       "      <th>feat_3</th>\n",
       "      <th>feat_4</th>\n",
       "      <th>feat_5</th>\n",
       "      <th>feat_6</th>\n",
       "      <th>feat_7</th>\n",
       "      <th>feat_8</th>\n",
       "      <th>feat_9</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_85</th>\n",
       "      <th>feat_86</th>\n",
       "      <th>feat_87</th>\n",
       "      <th>feat_88</th>\n",
       "      <th>feat_89</th>\n",
       "      <th>feat_90</th>\n",
       "      <th>feat_91</th>\n",
       "      <th>feat_92</th>\n",
       "      <th>feat_93</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.016393</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018182</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.013158</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.013158</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.016393</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.014286</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.131579</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.015385</td>\n",
       "      <td>0.029851</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018182</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id    feat_1  feat_2  feat_3    feat_4    feat_5  feat_6    feat_7  \\\n",
       "0   1  0.016393     0.0     0.0  0.000000  0.000000     0.0  0.000000   \n",
       "1   2  0.000000     0.0     0.0  0.000000  0.000000     0.0  0.000000   \n",
       "2   3  0.000000     0.0     0.0  0.000000  0.000000     0.0  0.000000   \n",
       "3   4  0.016393     0.0     0.0  0.014286  0.315789     0.1  0.131579   \n",
       "4   5  0.000000     0.0     0.0  0.000000  0.000000     0.0  0.000000   \n",
       "\n",
       "     feat_8  feat_9  ...   feat_85   feat_86   feat_87  feat_88  feat_89  \\\n",
       "0  0.000000     0.0  ...  0.018182  0.000000  0.000000      0.0      0.0   \n",
       "1  0.013158     0.0  ...  0.000000  0.000000  0.000000      0.0      0.0   \n",
       "2  0.013158     0.0  ...  0.000000  0.000000  0.000000      0.0      0.0   \n",
       "3  0.000000     0.0  ...  0.000000  0.015385  0.029851      0.0      0.0   \n",
       "4  0.000000     0.0  ...  0.018182  0.000000  0.000000      0.0      0.0   \n",
       "\n",
       "    feat_90  feat_91  feat_92  feat_93   target  \n",
       "0  0.000000      0.0      0.0      0.0  Class_1  \n",
       "1  0.000000      0.0      0.0      0.0  Class_1  \n",
       "2  0.000000      0.0      0.0      0.0  Class_1  \n",
       "3  0.000000      0.0      0.0      0.0  Class_1  \n",
       "4  0.007692      0.0      0.0      0.0  Class_1  \n",
       "\n",
       "[5 rows x 95 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('data/Otto_FE_train_org.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feat_1</th>\n",
       "      <th>feat_2</th>\n",
       "      <th>feat_3</th>\n",
       "      <th>feat_4</th>\n",
       "      <th>feat_5</th>\n",
       "      <th>feat_6</th>\n",
       "      <th>feat_7</th>\n",
       "      <th>feat_8</th>\n",
       "      <th>feat_9</th>\n",
       "      <th>feat_10</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_84</th>\n",
       "      <th>feat_85</th>\n",
       "      <th>feat_86</th>\n",
       "      <th>feat_87</th>\n",
       "      <th>feat_88</th>\n",
       "      <th>feat_89</th>\n",
       "      <th>feat_90</th>\n",
       "      <th>feat_91</th>\n",
       "      <th>feat_92</th>\n",
       "      <th>feat_93</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.016393</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.018182</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.013158</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.013158</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.016393</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.014286</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.131579</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>...</td>\n",
       "      <td>0.289474</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.015385</td>\n",
       "      <td>0.029851</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.018182</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 93 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     feat_1  feat_2  feat_3    feat_4    feat_5  feat_6    feat_7    feat_8  \\\n",
       "0  0.016393     0.0     0.0  0.000000  0.000000     0.0  0.000000  0.000000   \n",
       "1  0.000000     0.0     0.0  0.000000  0.000000     0.0  0.000000  0.013158   \n",
       "2  0.000000     0.0     0.0  0.000000  0.000000     0.0  0.000000  0.013158   \n",
       "3  0.016393     0.0     0.0  0.014286  0.315789     0.1  0.131579  0.000000   \n",
       "4  0.000000     0.0     0.0  0.000000  0.000000     0.0  0.000000  0.000000   \n",
       "\n",
       "   feat_9   feat_10  ...   feat_84   feat_85   feat_86   feat_87  feat_88  \\\n",
       "0     0.0  0.000000  ...  0.000000  0.018182  0.000000  0.000000      0.0   \n",
       "1     0.0  0.000000  ...  0.000000  0.000000  0.000000  0.000000      0.0   \n",
       "2     0.0  0.000000  ...  0.000000  0.000000  0.000000  0.000000      0.0   \n",
       "3     0.0  0.033333  ...  0.289474  0.000000  0.015385  0.029851      0.0   \n",
       "4     0.0  0.000000  ...  0.000000  0.018182  0.000000  0.000000      0.0   \n",
       "\n",
       "   feat_89   feat_90  feat_91  feat_92  feat_93  \n",
       "0      0.0  0.000000      0.0      0.0      0.0  \n",
       "1      0.0  0.000000      0.0      0.0      0.0  \n",
       "2      0.0  0.000000      0.0      0.0      0.0  \n",
       "3      0.0  0.000000      0.0      0.0      0.0  \n",
       "4      0.0  0.007692      0.0      0.0      0.0  \n",
       "\n",
       "[5 rows x 93 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 分离x 和y\n",
    "y_train = train['target']\n",
    "x_train = train.drop(['id','target'],axis =1)\n",
    "x_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存特征名字后边备用\n",
    "feat_names = x_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "x_train = csr_matrix(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.79764022 0.79738552 0.79737979]\n",
      "cv logloss is:  0.7974685091743186\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "# 3折交叉验证\n",
    "loss = cross_val_score(lr, x_train, y_train, cv = 3, scoring = 'neg_log_loss' )\n",
    "print('logloss of each fold is: ', -loss)\n",
    "print('cv logloss is: ',-loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='liblinear',\n",
       "                                          random_state=None, solver='warn',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'C': [0.1, 1, 10, 100, 1000], 'penalty': ['l1', 'l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用GridSearchCV调优\n",
    "# 设置参数搜索范围\n",
    "# 生成学习实例\n",
    "# 生成GridSearchCV实例\n",
    "# 调用GridSearchCV的fit方法\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.1,1,10,100,1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "lr_penalty = LogisticRegression('liblinear')\n",
    "grid = GridSearchCV(lr_penalty,tuned_parameters,cv = 3,scoring = 'neg_log_loss')\n",
    "grid.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.672850269307725\n",
      "{'C': 100, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# 打印分数 和参数\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mean_fit_time': array([2.23402794, 1.67839901, 2.77498579, 2.41620374, 3.15481575,\n",
      "       3.77957026, 3.12815229, 5.64246289, 3.37047704, 7.62104599]), 'std_fit_time': array([0.09805166, 0.06210767, 0.10188602, 0.09488999, 0.01825614,\n",
      "       0.09386662, 0.02340242, 0.46366413, 0.10019141, 0.50965896]), 'mean_score_time': array([0.16426508, 0.14536905, 0.14257232, 0.15150587, 0.14657021,\n",
      "       0.14058797, 0.14285151, 0.14678367, 0.16352924, 0.2100873 ]), 'std_score_time': array([0.0155364 , 0.00184764, 0.00987799, 0.01533102, 0.01119059,\n",
      "       0.01317459, 0.01854803, 0.00675251, 0.02137273, 0.09115892]), 'param_C': masked_array(data=[0.1, 0.1, 1, 1, 10, 10, 100, 100, 1000, 1000],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
      "                   'l2'],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 0.1, 'penalty': 'l1'}, {'C': 0.1, 'penalty': 'l2'}, {'C': 1, 'penalty': 'l1'}, {'C': 1, 'penalty': 'l2'}, {'C': 10, 'penalty': 'l1'}, {'C': 10, 'penalty': 'l2'}, {'C': 100, 'penalty': 'l1'}, {'C': 100, 'penalty': 'l2'}, {'C': 1000, 'penalty': 'l1'}, {'C': 1000, 'penalty': 'l2'}], 'split0_test_score': array([-0.86408058, -1.05293105, -0.70274349, -0.79764022, -0.67710405,\n",
      "       -0.70258371, -0.67592348, -0.67942163, -0.67608419, -0.67581226]), 'split1_test_score': array([-0.86371591, -1.05283464, -0.6994404 , -0.79738552, -0.67311454,\n",
      "       -0.69969151, -0.67169179, -0.67523915, -0.67175906, -0.67157473]), 'split2_test_score': array([-0.86152787, -1.05405918, -0.70036771, -0.79737979, -0.67359448,\n",
      "       -0.70081234, -0.67093505, -0.6762024 , -0.67085679, -0.67154141]), 'mean_test_score': array([-0.8631082 , -1.05327492, -0.70085061, -0.79746852, -0.67460447,\n",
      "       -0.70102924, -0.67285027, -0.6769545 , -0.67290019, -0.67297627]), 'std_test_score': array([0.00112725, 0.00055591, 0.00139107, 0.00012144, 0.00177842,\n",
      "       0.00119068, 0.00219509, 0.00178842, 0.00228153, 0.00200554]), 'rank_test_score': array([ 9, 10,  6,  8,  4,  7,  1,  5,  2,  3], dtype=int32)}\n"
     ]
    }
   ],
   "source": [
    "print(grid.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5dXA8d9JyMIS1oRFwr5DZTMiuIEiq624V6kVKxZt1Vprrah1Ke7L++rb93UBFVHbqtTWlloVF0C0iBCURZYg4BbWCELCGpKc9497Q4ZkZjIJuXMzM+f7+dzPzNz73DtnhiSH+zz3nkdUFWOMMaayJL8DMMYYUz9ZgjDGGBOUJQhjjDFBWYIwxhgTlCUIY4wxQTXwO4C6kpmZqZ07d/Y7DGOMiSnLli37TlWzgm2LmwTRuXNncnNz/Q7DGGNiioh8HWqbdTEZY4wJyhKEMcaYoCxBGGOMCSpuxiCMMSaUw4cPk5+fz8GDB/0OxTfp6elkZ2eTkpIS8T6WIIwxcS8/P5+MjAw6d+6MiPgdTtSpKjt37iQ/P58uXbpEvJ91MRlj4t7Bgwdp1apVQiYHABGhVatWNT6DsgRhjEkIiZocytXm83uWIERkpojsEJHPQ2wXEfmjiGwQkZUiMjhgW6mILHeXOV7FeMTXi+DQXs/fxhgTO348/WN+PP1jv8PwlZdnELOAsWG2jwN6uMsU4KmAbQdUdaC7nONdiMB3G+D5cfDkUE/fxhiT2Jo0aXLk+dixY2nevDk//OEPj6w777zzGDhwIN27d6dZs2YMHDiQgQMHsmjRohq9z7x581i8eHGdxOzZILWqLhSRzmGaTABeVGfGosUi0lxE2qnqVq9iCiqzOxx/Maz5J3z/NbToFNW3N8Yknptvvpn9+/czffr0I+tef/11ABYsWMCjjz7KG2+8Uatjz5s3j8zMTIYOPfb/9Po5BtEe+Dbgdb67DiBdRHJFZLGInBvqACIyxW2XW1BQUPtIzrobkpLh3TtrfwxjjInQyJEjycjIiLj90qVLGT58OCeccALjxo1j+/btADz22GP07duXAQMGcNlll7Fx40aeffZZHnnkkVqdfVTm52WuwUZMyuc/7aiqW0SkKzBPRFap6sYqjVVnADMAcnJyaj93arP2cOqNMP8++Ooj6HxqrQ9ljKnf/vCv1azZUlhtuzVbnTaRjEP0Pa4pd/2o3zHHFsyhQ4e44YYbmDNnDpmZmfz5z3/mjjvuYMaMGTz88MN8/fXXpKamsnv3bpo3b85VV11FZmYmv/71r4/5vf1MEPlAh4DX2cAWAFUtf9wkIguAQUCVBFGnTr4ePn0R3poKV3/gnFEYY4zP1q5dy+rVqznrrLMAKC0tJTs7G4B+/fpx2WWXMWHCBM49N2RnS635mSDmANeJyCvAScAeVd0qIi2A/ap6SEQygVOAhz2PJqUhjL4H/noFfPYSnHCF529pjIm+SP+nX37m8OrVw7wMp1qqSv/+/fnwww+rbJs7dy4ffPAB//znP7n33nv5/POgF43WmpeXub4MfAz0EpF8EZksIteIyDVukzeBTcAG4Bngl+76PkCuiKwA5gMPquoar+I8St9zoePJ8P49cGB3VN7SGGPC6du3L5s3b2bJkiUAFBcXs3r1akpLS8nPz+fMM8/kkUceoaCggP3795ORkUFRUVGdvLeXVzFdWs12Ba4Nsn4RcLxXcYUlAuMehOnDYeEjMOY+X8IwxsS30047jXXr1rF3716ys7N57rnnGDNmTNC2aWlpvPbaa/zqV7+iqKiIkpISbrrpJrp3787EiRMpKiqirKyMW265hYyMDCZMmMBFF13E3//+d5544glOPvnkWsdptZgqazcABl8OnzztdDNl9vA7ImNMHNi7t+Jm3GDdReVGjBjBiBEjjlo3ePBgPvrooypt//Of/1RZ17t3b1atWlX7QANYqY1gzrwDUhrB3Nv9jsQY45NXrx7m+/iD3yxBBNMkC4b/Dr6YC1+853c0xhjjC0sQoQy5Glp2g7m3Qulhv6MxxpioswQRSoNUGHM/fLcelj7rdzTGGBN1liDC6TkGuo2EBQ/Avp1+R2OMMVFlCSIcEecs4tBepwyHMSZxPH+2syQwSxDVad0bhvwclj0P2+r2LkVjTOIoL/e9fPlyhg0bRr9+/ejfvz+vvvoqkGDlvuPKiKmwcja8PRUm/cs5szDGmFpo1KgRL774Ij169GDLli2ccMIJjBkzxsp9x6yGLeCM2+CrD2Fd7f7RjDEGoGfPnvTo4dyAe9xxx9G6dWuqm64gEct9x5YTfga5M52b57qPgpR0vyMyxtTGW1NhWwR3Gm9b6TxGMg7R9ninTE8NLVmyhOLiYrp16xayTaKW+44tyQ1g7APw4gRY/CSc9hu/IzLGxLCtW7fy05/+lBdeeIGkpNCdOYla7jv2dB0BvX8ICx+FgRMho63fERljairS/+mXnzn87N91HkJhYSFnn3029957b7VjBXFZ7jtujb4Hyg7D+9P8jsQYE4OKi4s577zzuPzyy7nooouqbe9nuW9LEDXVsisM/SUs/zNsXuZ3NMaYGDN79mwWLlzIrFmzjlzKunz58pDty8t9/+Y3v2HAgAEMGjSITz75hJKSEiZOnEj//v0ZPHjwUeW+Z8+ezaBBg455kFqcaRliX05Ojubm5kbnzQ4Vwf+eAM07weR37LJXY+q5tWvX0qdPn5rt5GEXk1+CfQ8iskxVc4K1tzOI2kjLgJF3Qv4SWPWa39EYY7zws3/HVXKoDS+nHJ0pIjtEJOioiTj+KCIbRGSliAwO2DZJRL5wl0lexXhMBkyEdgPh3TuheJ/f0RhjTJ3z8gxiFjA2zPZxQA93mQI8BSAiLYG7gJOAIcBdItLCwzhrJykJxj0MRVvgo8f9jsYYU4146U6vrdp8fs8ShKouBHaFaTIBeFEdi4HmItIOGAO8q6q7VPV74F3CJxr/dDwJfnAhLPoj7P7G72iMMSGkp6ezc+fOhE0SqsrOnTtJT6/ZDb5+3gfRHvg24HW+uy7U+ipEZArO2QcdO3b0JsrqjPoDrPu309V00Sx/YjDGhJWdnU1+fn61JS3iWXp6+pEb7CLlZ4IIdumPhllfdaXqDGAGOFcx1V1oNdAsG069ERbcD0OmQKeTfQnDGBNaSkoKXbp08TuMmOPnVUz5QIeA19nAljDr66+Tr4em2fDWLVBW6nc0xhhTJ/xMEHOAy92rmYYCe1R1KzAXGC0iLdzB6dHuuvortRGMnuYU9/rsT35HY4wxdcKzLiYReRkYAWSKSD7OlUkpAKr6NPAmMB7YAOwHfuZu2yUi9wBL3UNNU9Vwg931Q7/zYckzMO8e6HcupDfzOyJjjDkmdid1XdryGcw4A06+Dkbf628sxhgTAbuTOlqOGwSDLoPFT8POjX5HY4wxx8QSRF0beSc0SHcmFjLGmBhmCaKuNWkNw2+G9W/Bhvf9jsYYY2rNEoQXTrrGKQs+9zYoPex3NMYYUyuWILzQIA1G3wcF65x5rI0xJgZZgvBKr3HQ9QyYfz/sr/9X6RpjTGWWILwiAmMfcCYXmn+/39EYY0yNWYLwUus+cOJkyH0Otq/2OxpjjKkRSxBeG3Grc1f127dCnNyUaIxJDJYgvNaoJYy4Db78APLe9DsaY4yJmCWIaMi5ErL6OJe9lhzyOxpjjImIJYhoSG4AY++H77+CxU/6HY0xxkTEEkS0dDsTeo2HhY9C0Xa/ozHGmGpZgoim0fc6XUzvT/M7EmOMqZYliGhq1Q2G/gKW/xk2f+p3NMYYE5YliGg7/WZonAlvT7XLXo0x9ZoliGhLbwoj74JvP4HP/+Z3NMYYE5KnCUJExopInohsEJGpQbZ3EpH3RWSliCwQkeyAbaUistxd5ngZZ9QN/Am0GwDv3gnF+/2OxhhjgvIsQYhIMvAEMA7oC1wqIn0rNXsUeFFV+wPTgAcCth1Q1YHuco5XcfoiKQnGPgSFm+E//+N3NMYYE5SXZxBDgA2quklVi4FXgAmV2vQFymfVmR9ke/zqNAz6nQ//eRx2f+t3NMYYU4WXCaI9EPiXL99dF2gFcIH7/DwgQ0Raua/TRSRXRBaLyLnB3kBEprhtcgsKCuoy9ugY5V7u+t5d/sZhjDFBeJkgJMi6ypft/BYYLiKfAcOBzUCJu62jquYAE4HHRaRblYOpzlDVHFXNycrKqsPQo6R5Bzjl185g9dcf+x2NMcYcxcsEkQ90CHidDWwJbKCqW1T1fFUdBNzurttTvs193AQsAAZ5GKt/TrkBmraHt2+BsjK/ozHGmCO8TBBLgR4i0kVEUoFLgKOuRhKRTBEpj+FWYKa7voWIpJW3AU4B1ngYq39SGzldTVtXODfQGWNMPeFZglDVEuA6YC6wFpitqqtFZJqIlF+VNALIE5H1QBvgPnd9HyBXRFbgDF4/qKrxmSAAfnABdBgK7/8BDhb6HY0xxgAgGid38+bk5Ghubq7fYdTe5k/hmTOcLqdRVqvJGBMdIrLMHe+twu6kri/aD4aBl8HHT8LOjX5HY4wxliDqlZF3QoM0eOf3fkdijDGWIOqVjDZw+m+dqUk3zvM7GmNMgrMEUd8M/SW06AJv3wqlJdW3N8YYj1iCqG8apDkTCxWsg9yZfkdjjElgliDqo95nQ5fhMP8+2L/L72iMMQnKEkR9JAJjH4BDhbDggerbG2OMByxB1Fdt+kHOlbD0Odix1u9ojDEJyBJEfXbG7ZCW4QxYx8kNjcaY2GEJoj5r1BJG3Aqb5kPeW35HY4xJMJYg6rsTJ0NmL3jndig55Hc0xpgEYgmivktOgbH3w65N8MnTfkdjjEkgliBiQfezoOdY+OAR2LvD72iMMQnCEkSsGH0flByE963SqzEmOixBxIrM7nDS1fDZn2DLcr+jMcYkAEsQsWT476BRK3h7ql32aozxnKcJQkTGikieiGwQkalBtncSkfdFZKWILBCR7IBtk0TkC3eZ5GWcMSO9mVMS/JuPYfXf/Y7GGBPnPEsQIpIMPAGMA/oCl4pI30rNHgVeVNX+wDTgAXfflsBdwEnAEOAuEWnhVawxZdBl0PZ4eOdOKN7vdzTGmDjm5RnEEGCDqm5S1WLgFWBCpTZ9gffd5/MDto8B3lXVXar6PfAuMNbDWGNHUjKMfQgK82HR//odjTEmjnmZINoD3wa8znfXBVoBXOA+Pw/IEJFWEe6buDqfAn3PhY8egz35fkdjjIlTXiYICbKu8sjqb4HhIvIZMBzYDJREuC8iMkVEckUkt6Cg4FjjjS2jpgEK797ldyTGmDjlZYLIBzoEvM4GtgQ2UNUtqnq+qg4CbnfX7YlkX7ftDFXNUdWcrKysuo6/fmvRCU7+FXz+Gnyz2O9ojDFxyMsEsRToISJdRCQVuASYE9hARDJFpDyGW4HyKdTmAqNFpIU7OD3aXWcCnfpryDgO3roFysr8jsYYE2c8SxCqWgJch/OHfS0wW1VXi8g0ETnHbTYCyBOR9UAb4D53313APThJZikwzV1nAqU2hlF/gK3LYcVf/I7GGBNnRGt4w5X7P/4mqlroTUi1k5OTo7m5uX6HEX2q8Nxo+P4ruH4ZpDf1OyJjTAwRkWWqmhNsW0RnECLyFxFpKiKNgTU4/+u/uS6DNLUkAuMehH074MP/8jsaY0wcibSLqa97xnAu8CbQEfipZ1GZmml/AgyYCIufdMqCG2NMHYg0QaSISApOgvinqh4myGWnxkcj74SkFHjnDr8jMcbEiUgTxHTgK6AxsFBEOgH1agwi4TVtB6ffBOvegE0L/I7GGBMHIkoQqvpHVW2vquPV8TVwhsexmZoaei007wRv3wqlJX5HY4yJcZEOUt/gDlKLiDwnIp8CZ3ocm6mplHQYfS/sWAPLnvc7GmNMjIu0i+lKd5B6NJAF/Ax40LOoTO31+RF0Pg3m3wf77dYRY0ztRZogymsjjQeeV9UVBK+XZPwmAmMfhIN74IOH/I7GGBPDIk0Qy0TkHZwEMVdEMgCr7VBftf0BnHAFLHkGdqzzOxpjTIyKNEFMBqYCJ6rqfiAVp5vJ1Fdn3A5pTWDurTY9qTGmViK9iqkMp6Lq70XkUeBkVV3paWTm2DTOhOFTYeM8WG91Do0xNRfpVUwPAjfglNlYA/xKRB7wMjBTB4b8HDJ7wtzboKTY72iMMTEm0i6m8cAoVZ2pqjNxpv8827uwomvNlkLKyuKwGyY5BcY8ALs2wpLpfkdjjIkxNSn33TzgebO6DsQvG3bs5ew/fsiwB9+nppVtY0KPs6DHaPjgYdi7w+9ojDExJNIE8QDwmYjMEpEXgGXA/d6FFT3dshoz+dQubC88xJMLNvodjjfG3A+H98O8e/yOxBgTQxpE0khVXxaRBcCJOPc/3KKq27wMLFpEhNvG92HnvmIemZtHq8apXDKko99h1a3MHjDkaqfa64lXQbsBfkdkjIkBYc8gRGRw+QK0w5kr+lvgOHddXEhKEh6+sD/De2Zx2+ureGd1XOS+ow3/HTRqCW9NtctejTERqa6L6b/CLI9Wd3ARGSsieSKyQUSmBtneUUTmi8hnIrJSRMa76zuLyAERWe4uT9f0g9VUSnIST102mOOzm3P9y5+x5Ms4K1PRsDmceQd8swjW/MPvaIwxMaDGU45GfGCRZGA9MArnzGMpcKmqrgloMwP4TFWfEpG+wJuq2llEOgNvqOoPIn2/uppydNe+Yi58ehEFRYeYffUw+rSLoyk8y0ph+nA4uBuuWwopDf2OyBjjs7qYcvT8IMtIEWkdZrchwAZV3aSqxcArwIRKbRQo/wvcDNgSSTxeatk4lZcmn0Tj1AZMmrmEb3ft9zukupOUDGMfgD3fwqL/9TsaY0w9V5NSG88CP3GXZ4DfAP8RkVBTj7bHGa8ol++uC3Q3cJmI5ONMZXp9wLYubtfTByJyWrA3EJEpIpIrIrkFBQURfpTqtW/ekBcnD+FQSRmXz1zCzr2H6uzYvutyGvSdAB89Bns2+x2NMaYeizRBlAF9VPUCVb0A6AscAk4CbgmxT7Bqr5X7sy4FZqlqNs7NeC+JSBKwFeioqoNwEtFfRKRKX4+qzlDVHFXNycrKivCjRKZnmwxmXpHD1j0H+Nmspew9FEcT8Iy6x+lueu9uvyMxxtRjkSaIzqq6PeD1DqCnqu4CDofYJx/oEPA6m6pdSJOB2QCq+jGQDmSq6iFV3emuXwZsBHpGGGudOaFTS56YOJjVWwq55qVlFJfESQHbFp3g5Oth1Wz4donf0Rhj6qlIE8SHIvKGiEwSkUnAHJy5qRsDu0PssxToISJdRCQVuMTdL9A3wEgAEemDkyAKRCTLHeRGRLoCPYBNNflgdWVknzY8eP7xfLThO27664r4Kclx6o3QpC28dQuUxUniM8bUqUgTxLXA88BAYBDwAnCtqu5T1aBzU6tqCXAdMBdYC8xW1dUiMk1EznGb3QT8XERWAC8DV6hzWdXpwEp3/WvANe7Zii8uyunA1HG9+deKLUx7Y018lORIawKj/gBbPoWVr/gdjTGmHor4MlcRaYNzZZICS1S1XhX2qavLXENRVe7791qe/ehLbh7Ti2vP6O7Ze0VNWRk8N8q5qun6ZZCW4XdExpgoq4vLXC8GlgAXAhcDn4jIhXUXYv1XXpLjvEHteWRuHq8s+cbvkI5dUhKMewj2bocP/9vvaIwx9UxEtZiA23Fmk9sBICJZwHs43T8Jo7wkx659xdz2+ipaNk5ldL+2fod1bLJzoP8l8PH/weDLoWUXvyMyxtQTkY5BJFXqUtpZg33jSlyW5DjrLkhKgXfv8DsSY0w9Eukf+bdFZK6IXCEiVwD/xrmxLSE1Sm3A81ecSPsWDZn8wlLWbi30O6Rj0/Q4OO1GWPsv2PSB39EYY+qJSOekvhmYAfQHBgAzVDXUDXIJIe5Kcgy7Dpp3hLdvhdI4uinQGFNrEXcTqerfVPU3qnqjqr7uZVCxIq5KcqQ0dO6w3rEaPn3B72iMMfVAdfNBFIlIYZClSERivF+lbsRVSY6+E6DTqTDvXjjwvd/RGGN8FjZBqGqGqjYNsmSoahzVwT42cVOSQ8Sp9npwNyx4yO9ojDE+S8grkbwQNyU52vV3Lndd+gwU5PkdjTHGR5Yg6lDclOQ48w5IaQxzb/M7EmOMjyxB1LGrT+/KVad2Ydair3hywUa/w6mdxpnOHNYb3oP17/gdjTHGJ5Yg6ljclOQYMgVadYe5t0JJsd/RGGN8YAnCA+UlOYb3zOK211fxzuptfodUcw1SYcwDsHMDLJnhdzTGGB9YgvBIXJTk6Dkaup8FHzwMe+tuSldjTGywBOGhuCjJMeZ+OLwP5t/rdyTGmCizBOGxmC/JkdXLGY9Y9gJsXel3NMaYKLIEEQUxX5Jj+O+gYQunTlOsXrprjKkxTxOEiIwVkTwR2SAiU4Ns7ygi80XkMxFZKSLjA7bd6u6XJyJjvIwzGmK6JEfDFnDm7+Hrj2DNP/2OxhgTJZ4lCBFJBp4AxgF9gUtFpG+lZr/Hmat6EHAJ8KS7b1/3dT9gLPCke7yYFtMlOQZPgtb94J074PABv6MxxkSBl2cQQ4ANqrpJVYuBV4AJldooUF7TqRmwxX0+AXhFVQ+p6pfABvd4MS9mS3IkN3DqNO35xpl9zhgT97xMEO2BbwNe57vrAt0NXCYi+TgTEF1fg30RkSkikisiuQUFsXMZZsyW5Og6HPr8yJm/es9mv6MxxnjMywQhQdZV/kt4KTBLVbOB8cBLIpIU4b6o6gxVzVHVnKysrGMOOJpitiTHqHug5CD8cZBTisMYE7e8TBD5QIeA19lUdCGVmwzMBlDVj4F0IDPCfWNazJbkaNkFrnwHWnSGP10Ar/8C9sfgTYDGmGp5mSCWAj1EpIuIpOIMOs+p1OYbYCSAiPTBSRAFbrtLRCRNRLoAPYAlHsbqi5gtydHhRLh6IZz2W1j5Kjw51JnP2hgTVzxLEKpaAlwHzAXW4lyttFpEponIOW6zm4Cfi8gK4GXgCnWsxjmzWAO8DVyrqqVexeqnmC3JkZIOI++AKfOhSWt49TKYPclKchgTRyRmBkirkZOTo7m5uX6HUWu79hVz4dOLKCg6xOyrh9GnXQxN2Fd6GP7zuFOzKbUJjHsIjr/ImaHOGFOvicgyVc0Jts3upK4nYrokR3IKnH4zXP0htOoGf/85vHwJFMbVsJExCccSRD0S8yU5WveGK+c6ZcI3fQBPnATLZll5DmNilCWIeiamS3IAJCXDsF/CLxdBuwHwrxvgxQnw/Vd+R2aMqSFLEPVQTJfkKNeyK1w+B374OGz+FJ4cBoufgrK4vNbAmLhkCaKeitmSHIGSkiDnZ3DtYuh8Krw9FZ4fBwXr/Y7MGBMBSxD1WMyW5KisWTZMnA3nTYfv1sPTp8KH/wWlMdZ9ZkyCsQRRz119elcmx2JJjspEYMAlcO0S6DUW3p8Gz54J21b5HZkxJgRLEPWciHB7LJbkCKVJa7j4RWcp3AozRsC8+6Akxq7YMiYBWIKIAZVLcsyNlZIc4fSdANd+4txQt/BhmH465MfujY7GxCNLEDGickmOTzbt9DukY9eoJZz3NEz8KxwqgudGwdzboTiGbhI0Jo5ZgoghjVIb8PwVJ5LdoiFXvZjL2q2FfodUN3qOhl8udmat+/j/4OlT4KuP/I7KmIRnCSLGxHRJjnDSm8KPHodJ/wItg1lnwxu/cc4sjDG+sAQRg2K+JEc4XU6HXyyCoddC7kx4Yih8YRMTGeMHSxAxKuZLcoST2hjG3g+T33We/9kmJjLGD5YgYlhclOQIp8OJcM2HFRMTPXGSTUxkTBRZgohxcVGSI5wGaRUTE2W0CZiYaIffkRkT9yxBxIG4KckRTrsB8PP5cOYdkPcmPDEEVs62UuLGeMjTBCEiY0UkT0Q2iMjUINsfE5Hl7rJeRHYHbCsN2FZ5LmtTSdyU5AgnOQVO/y1c8xG06uFMTPSXH8OezX5HZkxcauDVgUUkGXgCGAXkA0tFZI6qrilvo6o3BrS/HhgUcIgDqjrQq/jiTXlJjl37inlkbh6tGqdyyZCOfofljaxecOXb8Ml0p6bTk0Nh9D3OfRQ2zakxdcbLM4ghwAZV3aSqxcArwIQw7S8FXvYwnrgXlyU5Qgk6MdE5sOtLvyMzJm54mSDaA98GvM5311UhIp2ALsC8gNXpIpIrIotF5NwQ+01x2+QWFBTUVdwxLS5LcoTTsqtzc90PH4fNn8FTJ9vERMbUES8TRLBz/VAjipcAr6lq4G91R1XNASYCj4tItyoHU52hqjmqmpOVlXXsEceJuC3JEYpI1YmJZo61iYmMOUZeJoh8oEPA62xgS4i2l1Cpe0lVt7iPm4AFHD0+YaoRtyU5wjkyMdEM2PlFwMREh/2OzJiY5GWCWAr0EJEuIpKKkwSqXI0kIr2AFsDHAetaiEia+zwTOAVYU3lfE17lkhzfxVNJjlBEYMCPj56Y6JkzYetKvyMzJuZ4liBUtQS4DpgLrAVmq+pqEZkmIucENL0UeEWPvni/D5ArIiuA+cCDgVc/mcgFluS4Mt5KcoRzZGKil6BoGzxzBsy71yYmMqYGJF5uqsrJydHcXJtwJpT3125nykvLGNa1FTOvOJHUBgl0j+T+XTD3NljxMmT1hglPQHaO31EZUy+IyDJ3vLeKBPorkdjiviRHOOUTE/3kNZuYyJgasASRQBKiJEc4PUY5ExOdcIUzMdFTJ9vERMaEYQkiwSRESY5w0pvCDx+DSW84r2edDW/cCAfj/FJgY2rBEkSCKS/Jcd6g9jwyN49Xlnzjd0j+6HKaMzHRsOtg2Sx4cphNTGRMJZYgElBCleQIJ7URjLkPrnzHJiYyJghLEAkq4UpyhFM+MdHpN8Oq2c7ERGusgLAxliASWMKV5AinQRqc+XtnzomMtjD7pzYxkUl4liASXEKW5AinXX/4+TwYeWfFxEQrXrWJiUxCsgRhErMkRzjJKXDaTRUTE70+xSYmMgnJEoQBErgkRzjlExONfRC++tCZmCj3eTubMAnDEoQ54oROLXli4mBWbynkmpeWUVxS5ndI/ktKhqG/cC6JPW4gvPFrm5jIJAxLEHZELYAAAA8eSURBVOYoCV2SI5yWXeDyOfCj/7GJiUzCsARhqkj4khyhiDhlOq79BDqfFjAxUZ7fkRnjCUsQJqiEL8kRTrP2MPFVOP8Zm5jIxDVLECYoK8lRDRHof7E7MdF4m5jIxCVLECYkK8kRgSat4eIXbGIiE5csQZiwrCRHhPqe44xNHH8xLHwEpp8O+TaBlYltniYIERkrInkiskFEpgbZ/piILHeX9SKyO2DbJBH5wl0meRmnCc9KckSoUUs47yl3YqK9NjGRiXmeTTkqIsnAemAUkA8sBS4NNbe0iFwPDFLVK0WkJZAL5AAKLANOUNXvQ72fTTnqvc27D3DBk4soU2XahH70bdeM7BYNSUoSv0Orfw4Wwnt3Q+5z0KILnHQNtO7tTHnapI0zhmFMPRBuylEvE8Qw4G5VHeO+vhVAVR8I0X4RcJeqvisilwIjVPVqd9t0YIGqvhzq/SxBRMf67UWM/58PKXHvj2icmkzPthn0btuU3m0z3KUpzRql+BxpPfHVR/CnC6HkQMW6tGbOXdpZvZyEUf68aTYkWa+via5wCaKBh+/bHvg24HU+cFKwhiLSCegCzAuzb/sg+00BpgB07Njx2CM21erZJoNVd48hb3sR67YWsm5bEeu2FfLW51t5OeBKp3bN0undNoNebZvSp10Gvdpm0DWzCakNEuwPYOdT4fatTlXYgnXw3XrnsSAP1r8Nn71U0TalMWT1dJJGZs+K5NGis3NHtzFR5mWCCHYOHep05RLgNVUtvy01on1VdQYwA5wziNoEaWquYWoyAzs0Z2CH5kfWqSo7ig6xdmsheduK3MRRxEcbNnG41PmnSUkWumU1OZI4erdzzjjaNk1H4rnLRQQy2jhL1+FHb9u/y0kW5UmjYB18uRBWBJwsJ6dBZo+jzzgye0HLrtAgNbqfxSQULxNEPtAh4HU2sCVE20uAayvtO6LSvgvqMDZTx0SENk3TadM0nRG9Wh9Zf7i0jE0F+1i3zTnbyNtWxJIvd/GP5RU/Cs0aptCrbQZ92mbQu11TerXNoFebDBqnefnjWU80agmdhjlLoIN74Lsv3MSxDgrWO1dFff63ijZJDaBlt6pdVa16QEp6dD+HiUtejkE0wBmkHglsxhmknqiqqyu16wXMBbqoG4w7SL0MGOw2+xRnkDrkXJA2BhFb9uw/TN72IvK2FbJ2m9NdlbetiH3FFbWNOrZsVDGu4SaOzq0ak5zIg+LF+5zEEdhVVbAOdm0CdYsrSpLTLZVZaZwjsyekNfE1fFP/+DIGoaolInIdzh//ZGCmqq4WkWlArqqWz+l4KfCKBmQqVd0lIvfgJBWAaeGSg4k9zRqlMKRLS4Z0aXlkXVmZsnn3Aad7amsh69xxjvfWbqe8ZmB6ShI9WlckjfIE0qpJmk+fJMpSGztVZY8bePT6kkOwc+PRSeO79bDhPSgLKAHSrEPVrqqsXtCwOcZU5tkZRLTZGUT8Oni4lA079lYkDnd8I3Bio8wmac5geJuKxNG9dRPSUxJ8cLe0BL7/8uiuqvLkUXKwol2TtgGJo3yAvDc0zvQvdhMVvlzmGm2WIBLPd3sPkbet6KiB8fXbizjkzmORnCR0btXISRgBiSO7RcP4HhSPRFkp7P6maldVQR4U761o16hV1a6qrF6Q0c7u5YgTliBMwigtU77auY91WyvGN/K2FfFNwFzbTdIaOAPh7sB4r7bO+EazhnbvBqpQuKVq0ihYBwd3V7RLa3r0vRzlSaRZB7uXI8ZYgjAJb++hEtZvL6qSOPYcqOifP65Z+pHB8PIb/rpmNSYl2f7goQr7CgISRkDy2Lejol1Ko4B7OAK6quxejnrLEoQxQagq2woPsm5r0ZEb/vK2FbFhx94jd4qX37vRx+2e6tU2gz7tmtI6I826qcrt3xW8q6pwc0Wb5DRo1b1qV1XLbnYvh88sQRhTA8UlZWz6bi/rthax1k0a67YWsa2wYlC3eaOUI2cZ5YmjV9sMGqUmwL0bkTpYWOlejjz4Lg++/5oj971KMrTqFnBFlZs8mrRxEkdyGjRIs7MPD1mCMKYO7N5ffORmv8Ab//a7926IBN670fTIpbgdWzZK7Hs3Kive78zEV7mratcm0BBzfEuykyiSU93HtIAEkuqsD7ktrdK+KUHWhdk3OTX48eIkaVmCMMYjZWVK/vcHWLut0Bnf2O48frVz35F7N5IEGiQlkZWRRnKSkJwkJAnuowSsc5+LkJSEuz6J5Eptk9w2Fesqba9yTNxjCg2C7h+4jqAxVaw7entyyJiOblv95xSSS4tJ3v0l8tcroKwETpzs3N9RWlzxGPj8qMdDUFLsPJYerrqu/LGspO7+8d2kpVWSTGqQxBO4Le3o58kpaIgkpUclqVDHToOUdJJTG9buY1iCMCa6DhSX8sUOZ2zjv9/Jo7RMOa1nFmVlSqk6iaW0TClVddc5r8tUKSl1Hksrta1YV7FPWRnBjxOwvaSs7EiyiiWRnHTV5GOpglBGKiXucth5lMNHXqdxmFSp2JZS3k5KnG0h9nNeBzzn8JH9nOM6jynB2kmIs6YaWFfWgd7TPq/Vvn5VczUmYTVMTaZ/dnP6Zzfn4pwO1e/gMVWlTAmeaAKTTZV1gUmHI89LQyWtI+tCH6tiHUcdv3z56zKnkPMFg7Mj+mw16ryrwYUF1bU87C77anZYpPKRtYxkPUxy2WGSy4ornmsxDbS40vriI8+T3DYHvliIJEHvyEOImCUIYxKAiBzpqqrvbhzV0+8QYsytnh3ZLvA2xhgTlCUIY4wxQVmCMMYYE5QlCGOMMUFZgjDGGBOUJQhjjDFBWYIwxhgTlCUIY4wxQVmCMMYYE1Tc1GISkQLg62M4RCbwXR2FU5csrpqxuGrG4qqZeIyrk6pmBdsQNwniWIlIbqiCVX6yuGrG4qoZi6tmEi0u62IyxhgTlCUIY4wxQVmCqDDD7wBCsLhqxuKqGYurZhIqLhuDMMYYE5SdQRhjjAnKEoQxxpigEjZBiMhFIrJaRMpEJOTlYSIyVkTyRGSDiEyNQlwtReRdEfnCfWwRol2piCx3lzkexhP284tImoi86m7/REQ6exVLDWK6QkQKAr6fq7yOyX3fmSKyQ0SCTg4sjj+6ca8UkcH1JK4RIrIn4Pu6M0pxdRCR+SKy1v1dvCFIm6h/ZxHGFfXvTETSRWSJiKxw4/pDkDZ1+/uoqgm5AH2AXsACICdEm2RgI9AVSAVWAH09juthYKr7fCrwUIh2e6PwHVX7+YFfAk+7zy8BXq0HMV0B/J8PP1OnA4OBz0NsHw+8hTPd8VDgk3oS1wjgDR++r3bAYPd5BrA+yL9l1L+zCOOK+nfmfgdN3OcpwCfA0Ept6vT3MWHPIFR1rarmVdNsCLBBVTepajHwCjDB49AmAC+4z18AzvX4/cKJ5PMHxvsaMFKkJlO4exKTL1R1IbArTJMJwIvqWAw0F5F29SAuX6jqVlX91H1eBKwF2ldqFvXvLMK4os79Dva6L1PcpfJVRnX6+5iwCSJC7YFvA17n4/0PShtV3QrODyrQOkS7dBHJFZHFIuJVEonk8x9po6olwB6glUfxRBoTwAVul8RrItLBw3hqwo+fp0gNc7su3hKRftF+c7crZBDO/4oD+fqdhYkLfPjORCRZRJYDO4B3VTXk91UXv48NartjLBCR94C2QTbdrqr/jOQQQdYd83XB4eKqwWE6quoWEekKzBORVaq68VhjqySSz+/JdxRGJO/3L+BlVT0kItfg/I/qTA9jilS0v6tIfYpTj2eviIwH/gH0iNabi0gT4G/Ar1W1sPLmILtE5TurJi5fvjNVLQUGikhz4HUR+YGqBo4t1en3FdcJQlXPOsZD5AOB//vMBrYc4zHDxiUi20WknapudU+ld4Q4xhb3cZOILMD5X05dJ4hIPn95m3wRaQA0w9vujGpjUtWdAS+fAR7yMJ6a8OTn6VgF/vFT1TdF5EkRyVRVz4vSiUgKzh/hP6vq34M08eU7qy4uP78z9z13u7/3Y4HABFGnv4/WxRTeUqCHiHQRkVScQR/PrhhyzQEmuc8nAVXOdESkhYikuc8zgVOANR7EEsnnD4z3QmCeuiNkHqk2pkp91Ofg9CHXB3OAy90rc4YCe8q7E/0kIm3L+6lFZAjO34Wd4feqk/cV4Dlgrar+d4hmUf/OIonLj+9MRLLcMwdEpCFwFrCuUrO6/X2M5ih8fVqA83Cy7SFgOzDXXX8c8GZAu/E4VzFsxOma8jquVsD7wBfuY0t3fQ7wrPv8ZGAVzhU8q4DJHsZT5fMD04Bz3OfpwF+BDcASoGsUvqPqYnoAWO1+P/OB3lH6mXoZ2Aocdn+2JgPXANe42wV4wo17FSGunvMhrusCvq/FwMlRiutUnO6PlcBydxnv93cWYVxR/86A/sBnblyfA3e66z37fbRSG8YYY4KyLiZjjDFBWYIwxhgTlCUIY4wxQVmCMMYYE5QlCGOMMUFZgjCmBkRkb/Wtwu7/mnv3OyLSRESmi8hGtzrnQhE5SURS3edxfSOrqf8sQRgTJW69nmRV3eSuehbnLtceqtoPpwptpjpFCN8HfuxLoMa4LEEYUwvunb2PiMjnIrJKRH7srk9yyy6sFpE3RORNEbnQ3e0nuHfGi0g34CTg96paBk7ZFFX9t9v2H257Y3xjp7DG1M75wEBgAJAJLBWRhThlTzoDx+NU4l0LzHT3OQXnrmaAfsBydYqvBfM5cKInkRsTITuDMKZ2TsWpGFuqqtuBD3D+oJ8K/FVVy1R1G06pj3LtgIJIDu4mjmIRyajjuI2JmCUIY2on1CQs4SZnOYBTKwecOj4DRCTc72AacLAWsRlTJyxBGFM7C4EfuxO4ZOFM67kE+AhnsqIkEWmDMzVlubVAdwB15u7IBf4QUBW0h4hMcJ+3AgpU9XC0PpAxlVmCMKZ2XsepqrkCmAf8zu1S+htOxdTPgek4M5Htcff5N0cnjKtwJo7aICKrcOauKJ/r4AzgTW8/gjHhWTVXY+qYiDRRZ6axVjhnFaeo6ja3hv9893WowenyY/wduFWrnzfdGM/YVUzG1L033IldUoF73DMLVPWAiNyFM2/wN6F2didC+oclB+M3O4MwxhgTlI1BGGOMCcoShDHGmKAsQRhjjAnKEoQxxpigLEEYY4wJ6v8BYNztNqr0MPAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_means = grid.cv_results_['mean_test_score']\n",
    "test_stds = grid.cv_results_['std_test_score']\n",
    "# train_means = grid.cv_results_['mean_train_score']\n",
    "# train_stds = grid.cv_results_['std_train_score']\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "# train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "# train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i,value in enumerate(penaltys):\n",
    "    plt.errorbar(x_axis,-test_scores[:,i],yerr = test_stds[:,i],label = penaltys[i]+'Test')\n",
    "#     plt.errorbar(x_axis,-train_scores[:,i],yerr = train_stds[:,i],label = penaltys[i]+'Train')\n",
    "plt.legend()\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('logloss')\n",
    "plt.savefig('LogisticGridSearchCV_c.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#把模型存起来下次要用就直接用了\n",
    "import _pickle as cPickle\n",
    "cPickle.dump(grid.best_estimator_,open('data/Otto_L1_org.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
